{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature: Fuzzy String Matching"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate edit distances between each question pair (Levenshtein, Jaro, Jaro-Winkler, ...)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This utility package imports `numpy`, `pandas`, `matplotlib` and a helper `kg` module into the root namespace."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pygoose import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fuzzy matching libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fuzzywuzzy import fuzz\n",
    "from jellyfish import jaro_distance, jaro_winkler"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Config"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Automatically discover the paths to various data folders and compose the project structure."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "project = kg.Project.discover()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Identifier for storing these features on disk and referring to them later."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_list_id = 'fuzzy'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Preprocessed and tokenized questions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens_train = kg.io.load(project.preprocessed_data_dir + 'tokens_spellcheck_train.pickle')\n",
    "tokens_test = kg.io.load(project.preprocessed_data_dir + 'tokens_spellcheck_test.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens = tokens_train + tokens_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_fuzzy_distances(pair):\n",
    "    q1_tokens, q2_tokens = pair\n",
    "    q1_text = ' '.join(pair[0])\n",
    "    q2_text = ' '.join(pair[1])\n",
    "\n",
    "    fuzzy_distances = np.array([\n",
    "        fuzz.ratio(q1_tokens, q2_tokens),\n",
    "        fuzz.partial_ratio(q1_tokens, q2_tokens),\n",
    "        fuzz.token_sort_ratio(q1_tokens, q2_tokens),\n",
    "        fuzz.token_set_ratio(q1_tokens, q2_tokens),\n",
    "        fuzz.partial_token_sort_ratio(q1_tokens, q2_tokens),\n",
    "    ], dtype='float')\n",
    "    \n",
    "    # Normalize to [0 - 1] range.\n",
    "    fuzzy_distances /= 100\n",
    "    \n",
    "    jelly_distances = np.array([\n",
    "        jaro_distance(q1_text, q2_text),\n",
    "        jaro_winkler(q1_text, q2_text),\n",
    "    ])\n",
    "    \n",
    "    return np.concatenate([fuzzy_distances, jelly_distances])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batches: 100%|██████████| 2751/2751 [23:49<00:00,  1.92it/s]\n"
     ]
    }
   ],
   "source": [
    "features = kg.jobs.map_batch_parallel(\n",
    "    tokens,\n",
    "    item_mapper=get_fuzzy_distances,\n",
    "    batch_size=1000,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = np.array(features[:len(tokens_train)])\n",
    "X_test = np.array(features[len(tokens_train):])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train: (404290, 7)\n",
      "X_test:  (2345796, 7)\n"
     ]
    }
   ],
   "source": [
    "print('X_train:', X_train.shape)\n",
    "print('X_test: ', X_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_names = [\n",
    "    'fuzz_ratio',\n",
    "    'fuzz_partial_ratio',\n",
    "    'fuzz_token_sort_ratio',\n",
    "    'fuzz_token_set_ratio',\n",
    "    'fuzz_partial_token_sort_ratio',\n",
    "    \n",
    "    'jaro',\n",
    "    'jaro_winkler',\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "project.save_features(X_train, X_test, feature_names, feature_list_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
